#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time: 2018/1/4 下午2:32
@Author: CZC
@File: parse_base_worker.py
"""
import json
import traceback

from bdp.i_crawler.i_extractor.ttypes import ExtractInfo, ExStatus, CrawlInfo, BaseInfo, PageParseInfo
from common import util
from common.annual_field import AnnualReports
from common.connectdb import MongoDB
from common.global_field import InfoType, Model, AnnualReportModel, PageCrawlError
from common.gsxt_field import GsModel
from common.logger import global_log
from common.queue_mq_thread import MqQueueThread
from config.beanstalk_config import test_merge_mq_conf
from config.common_config import province_chinese_dict, BaseConfig
from config.flag import *
from config.mongodb_config import AicDB
from get_annual_report_field_info import GetAnnualReportFieldInfo
from get_gsxt_field_info import GetGsxtFieldInfo


class ParseBaseWorker(GetGsxtFieldInfo, GetAnnualReportFieldInfo):
    def __init__(self, **kwargs):
        super(ParseBaseWorker, self).__init__()

        self.log = global_log
        # 初始化配置信息
        config = BaseConfig()
        self.host = config.host
        self.logfile = config.logfile
        self.target_table = config.target_table
        self.annual_report_table = config.annual_report_table
        self.is_nb_mq_open = config.is_nb_mq_open
        self.is_gs_mq_open = config.is_gs_mq_open
        self.gs_topic = config.gs_topic
        self.gs_nb_topic = config.gs_nb_topic

        # 指向发实体解析消息队列
        self.merge_mq = MqQueueThread(
            server_conf=test_merge_mq_conf, log=global_log)
        # 指向数据库
        self.target_db = MongoDB(AicDB)

    # 判断实体对象是否抓取正常
    @staticmethod
    def get_crawl_page(field_item, multi=False, part=InfoType.type_list):
        """
        :param field_item:
        :param multi:
        :param part:
        :return:TRUE 返回对象数组 调用者如果需要获取页面需要单独处理,FALSE 返回具体的页面信息
        """
        if not field_item:
            return field_item

        page_list = field_item.get(part)
        # 如果是详情页则允许为None
        if part == InfoType.type_detail and not page_list:
            return page_list

        if not isinstance(page_list, list) \
                or len(page_list) <= 0:
            raise PageCrawlError("未抓取到网页,或者抓取到无效或者失败的网页")

        if len(page_list) <= 0:
            raise PageCrawlError("未抓取到网页,或者抓取到无效或者失败的网页")

        if not multi:
            return page_list[0].get(u'text')

        return page_list

    def query_offline_task(self, item):

        if not isinstance(item, dict):
            self.log.info('参数错误: item = {item}'.format(item=item))
            return FAIL

        company = item.get('_id', None)
        if company is None:
            self.log.error('没有company_name字段: item = {item}'.format(item=item))
            return FAIL

        province = item.get('province')
        if not province:
            self.log.error('没有省份信息: company = {}'.format(company))
            return FAIL

        self.log.info('开始解析任务...province = {province} company = {company}'.format(
            province=province, company=company))
        try:
            status = self.query_company(item)
        except Exception as e:
            self.log.error('解析异常...')
            self.log.exception(e)
            status = FAIL

        self.log.info('完成解析任务...province = {province} company = {company} status = {status}'.format(
            province=province, company=company, status=status))
        return status

    # 开始解析
    def query_company(self, item):

        # 先判断抓取页面信息中是否有无效拦截页面
        # self.filter_captcha_page(item)
        if not item:
            return FAIL

        # 获得企业名称
        company = item.get('_id')
        province = item.get('province')
        province = province_chinese_dict[province]

        base_info = item.get('__{}'.format(Model.BASE_INFO))
        if base_info is None:
            self.log.error('没有基本信息: province = {province} company = {company}'.format(
                province=province, company=company))
            return FAIL

        base_info_url = self.__get_base_info_url(base_info)
        self.host = util.get_url_info(base_info_url).get('site')

        # 生成基本的in_time，u_time，省份，企业等字段
        base_info_dict = self.__build_base_info_dict(item, company, province)

        # 解析年报
        nb_flag, annual_report_list = self.parse_annual_report_info(base_info_dict, base_info_url, item)

        # 解析工商
        gs_flag = self.parse_gs_info(base_info_dict, base_info_url, item, annual_report_list)

        if gs_flag and nb_flag:
            return SUCCESS

        if not gs_flag:
            self.log.error('工商信息解析失败: province = {province} company = {company}'.format(
                company=company, province=province))
        if not nb_flag:
            self.log.error('年报信息解析失败: province = {province} company = {company}'.format(
                company=company, province=province))
        return FAIL

    # 解析年报
    def parse_annual_report_info(self, base_info_dict, base_info_url, item):
        annual_report_list = []
        # 年报信息
        company = base_info_dict.get('company')
        province = base_info_dict.get('province')
        in_time = base_info_dict.get('_in_time')

        # 开始解析
        all_flag = True
        for ear_year_all_info_dict in self.__parse_annual_report_model(company, province, item):
            all_info_dict = dict()
            all_info_dict.update(base_info_dict)
            all_info_dict.update(ear_year_all_info_dict)

            # # 判断解析属性个数是否符合要求
            # len_annual_report_info = len(all_info_dict)
            # if len_annual_report_info < MIN_NB_FIELD_NUM:
            #     raise StandardError('province = {province} company = {company} field len = {length} 年报字段过少'.format(
            #         company=company, length=len_annual_report_info, province=province))

            # 存储解析信息
            all_info_dict['company'] = company
            annual_report_list.append(all_info_dict)

            download_time = util.get_change_stamp(in_time)
            flag = self.__store_annual_report_model(base_info_url, download_time, all_info_dict)
            all_flag = all_flag and flag
        return all_flag, annual_report_list

    # 解析工商信息
    def parse_gs_info(self, base_info_dict, base_info_url, item, annual_report_list):
        all_info_dict = dict()
        all_info_dict.update(base_info_dict)
        company = all_info_dict.get('company')
        province = all_info_dict.get('province')
        in_time = all_info_dict.get('_in_time')

        # 开始解析
        all_info_dict.update(self.__parse_model(company, province, item))

        # # 判断解析属性个数是否符合要求
        # if len(all_info_dict) < MIN_GS_FIELD_NUM:
        #     raise StandardError('province = {province} company = {company} field len = {length} 工商字段过少'.format(
        #         company=company, length=len(all_info_dict), province=province))

        # 统一社会信用号处理
        self.__process_register_code(company, province, all_info_dict)

        # 存储股东信息 与 年份信息 以及 公布日期信息
        self.__process_annual_report_info(annual_report_list, all_info_dict)

        # 存储解析信息
        download_time = util.get_change_stamp(in_time)
        return self.__store_model(company, province, base_info_url, download_time, all_info_dict)

    # 初始化存储dict
    @staticmethod
    def __build_base_info_dict(item, company, province):
        u_time = util.get_now_time()
        in_time = item.get('_in_time')
        if in_time is None:
            in_time = u_time

        all_info_dict = dict(company=company,
                             _utime=u_time,
                             _in_time=in_time,
                             province=province)
        return all_info_dict

    # 解析工商各个模块
    def __parse_model(self, company, province, item):
        all_info_dict = {}

        for key, value in item.items():
            if 'info' not in key:
                continue
            if AnnualReportModel.ANNUAL_REPORT in key:
                continue

            field_info = key.replace('__', '')
            try:
                field_info_dict = eval('self.get_' + field_info)(value)
            except Exception as e:
                self.log.error('province:{province},company:{company},error-part:{value},error-info:{error}'.format(
                    company=company, error=traceback.format_exc(), province=province, value=value))
                self.log.exception(e)
                field_info_dict = {}

            all_info_dict.update(field_info_dict)

        return all_info_dict

    # 存储 同时发送到消息队列
    def __store_model(self, company, province, base_info_url, download_time, all_info_dict):
        flag = False
        if self.is_gs_mq_open:
            entity_extract_data = self.get_entity_extractor_info(company, base_info_url, download_time, all_info_dict,
                                                                 self.gs_topic)
            flag = self.merge_mq.push_sync_msg(entity_extract_data)
            self.log.info('发送工商到消息队列: province = {province} company = {company}'.format(
                company=company, province=province))
        else:  # 存储到数据库中
            try:
                self.target_db.find_and_modify(self.target_table,
                                               query={'company': company},
                                               update={'$set': all_info_dict},
                                               upsert=True)
                self.log.info('存储到数据库: province = {province} company = {company}'.format(
                    company=company, province=province))
                flag = True
            except Exception as e:
                self.log.error('存储工商信息到数据库失败')
                self.log.exception(e)
        return flag

    # 解析年报各个模块
    def __parse_annual_report_model(self, company, province, item):
        # 获得按年份划分好的抓取数据
        year_info_dict = self.__get_annual_report_year_info_dict(item)

        for year, year_info in year_info_dict.items():
            all_info_dict = dict(year=year)
            for key, value in vars(AnnualReportModel).items():
                if 'info' not in key:
                    continue

                field_info = year_info.get(value)
                try:
                    field_info_dict = eval('self.get_' + AnnualReportModel.ANNUAL_REPORT + key)(field_info)
                except Exception as e:
                    self.log.error(
                        'year:{year},province:{province},company:{company},error-part:{value},error-info:{error}'.format(
                            year=year, company=company, error=traceback.format_exc(), province=province, value=value))
                    self.log.exception(e)
                    field_info_dict = {}

                all_info_dict.update(field_info_dict)

            yield all_info_dict

    # 存储年报 同时发送到消息队列
    def __store_annual_report_model(self, base_info_url, download_time, all_info_dict):
        flag = False
        company = all_info_dict.get('company')
        province = all_info_dict.get('province')
        year = all_info_dict.get('year')
        if self.is_nb_mq_open:
            entity_extract_data = self.get_entity_extractor_info(company, base_info_url, download_time, all_info_dict,
                                                                 self.gs_nb_topic, year=year)
            flag = self.merge_mq.push_sync_msg(entity_extract_data)
            self.log.info('发送年报到消息队列: province = {province} year={year} company = {company}'.format(
                company=company, year=year, province=province))
        else:  # 存储到数据库中
            try:
                self.target_db.find_and_modify(self.annual_report_table,
                                               query={'company': company, 'year': year},
                                               update={'$set': all_info_dict},
                                               upsert=True)
                self.log.info('存储年报到数据库: province = {province} year = {year} company = {company}'.format(
                    company=company, year=year, province=province))
                flag = True
            except Exception as e:
                self.log.error('存储年报到数据库失败')
                self.log.exception(e)
        return flag

    def get_entity_extractor_info(self, company, base_info_url, download_time, all_info_dict, topic, year=None):

        # 去除none值
        store_model = util.del_none(all_info_dict)

        base_url = base_info_url.encode('utf-8')
        replace_company = company.replace('(', '（').replace(')', '）')
        if year is None:
            record = '|' + replace_company
        else:
            record = '|' + replace_company + '|' + str(year)
        _site_record_id = util.get_md5(record)

        if year is None:
            self.log.info('company = {company} record_id = {_set_record_id} url = {url}'.
                          format(company=company, _set_record_id=_site_record_id,
                                 url=base_url))
        else:
            self.log.info('company = {company} year = {year} record_id = {_set_record_id} url = {url}'.
                          format(company=company, _set_record_id=_site_record_id,
                                 url=base_url, year=year))

        store_model['_src'] = []
        store_model['_src'].append({'url': base_url, 'site': self.host, 'download_time': download_time})
        store_model['_site_record_id'] = _site_record_id

        extract_info = ExtractInfo()
        extract_info.ex_status = ExStatus.kEsSuccess
        extract_info.extract_data = json.dumps(store_model)
        extract_info.topic_id = topic

        crawl_info = CrawlInfo()
        crawl_info.content = ""
        crawl_info.download_time = download_time

        url_info = util.get_url_info(base_url)

        base_info = BaseInfo()
        base_info.site = url_info.get('site', '')
        base_info.url = url_info.get('url', '')
        base_info.site_id = url_info.get('site_id', 0)
        base_info.url_id = url_info.get('url_id', 0)

        return PageParseInfo(extract_info=extract_info, crawl_info=crawl_info, base_info=base_info)

    # 获得基本信息url
    @staticmethod
    def __get_base_info_url(base_info):
        base_info_list = base_info.get(InfoType.type_list)
        if base_info_list is None:
            return ''
        if not isinstance(base_info_list, list):
            return ''
        return base_info_list[0].get('url', '')

    # 获得按年份划分好的抓取数据
    @staticmethod
    def __get_annual_report_year_info_dict(item):
        year_info_dict = {}
        for field_name, field_info in item.items():
            if AnnualReportModel.ANNUAL_REPORT not in field_name:
                continue
            if '#' not in field_name:
                continue
            try:
                year = field_name.split('#')[1]
            except IndexError:
                raise IndexError('annual report filed error, company:{company}'.format(company=item.get('_id')))
            if year not in year_info_dict.keys():
                year_info_dict[year] = list()
            year_info_dict[year].append(field_info)
        return year_info_dict

    # 处理股东信息 公布日期 年份信息
    @staticmethod
    def __process_annual_report_info(annual_report_list, all_info_dict):
        annual_report_info = []
        for nb_model in annual_report_list:
            year = nb_model.get('year')
            publish_date = nb_model.get(AnnualReports.PUBLISH_DATE, '')
            if publish_date == '':
                continue
            shareholder_information = nb_model.get(AnnualReports.SHAREHOLDER_INFO, [])

            item = {
                'year': year,
                'publish_date': publish_date,
                'shareholder_information': shareholder_information
            }
            annual_report_info.append(item)
        all_info_dict[GsModel.ANNUAL_REPORT] = annual_report_info

    # 处理统一社会信用号
    def __process_register_code(self, company, province, all_info_dict):

        if GsModel.CODE not in all_info_dict:
            all_info_dict[GsModel.CODE] = ""
        if GsModel.REGISTERED_CODE not in all_info_dict:
            all_info_dict[GsModel.REGISTERED_CODE] = ""
        if GsModel.UNIFIED_SOCIAL_CREDIT_CODE not in all_info_dict:
            all_info_dict[GsModel.UNIFIED_SOCIAL_CREDIT_CODE] = ""

        # 去空格处理
        all_info_dict[GsModel.CODE] = all_info_dict[GsModel.CODE].strip()
        all_info_dict[GsModel.REGISTERED_CODE] = all_info_dict[GsModel.REGISTERED_CODE].strip()
        all_info_dict[GsModel.UNIFIED_SOCIAL_CREDIT_CODE] = all_info_dict[GsModel.UNIFIED_SOCIAL_CREDIT_CODE].strip()

        if all_info_dict[GsModel.CODE] == "" \
                and all_info_dict[GsModel.REGISTERED_CODE] == "" \
                and all_info_dict[GsModel.UNIFIED_SOCIAL_CREDIT_CODE] == "":
            self.log.error("当前企业没有统一社会信用号或者注册号: {province} {company}".format(
                company=company,
                province=province
            ))
            return

        # 有统一社会信用号
        if all_info_dict[GsModel.UNIFIED_SOCIAL_CREDIT_CODE] != "" \
                and all_info_dict[GsModel.CODE] == "" \
                and all_info_dict[GsModel.REGISTERED_CODE] == "":
            if len(all_info_dict[GsModel.UNIFIED_SOCIAL_CREDIT_CODE]) != 18:
                self.log.error("统一社会信用号不为18位: {province} {company}".format(
                    company=company, province=province))
            return

        # 有注册号
        if all_info_dict[GsModel.UNIFIED_SOCIAL_CREDIT_CODE] == "" \
                and all_info_dict[GsModel.CODE] == "" \
                and all_info_dict[GsModel.REGISTERED_CODE] != "":
            return

        # 如果有code 但是没有统一社会信用号 和注册号
        if all_info_dict[GsModel.CODE] != "" \
                and all_info_dict[GsModel.UNIFIED_SOCIAL_CREDIT_CODE] == "" \
                and all_info_dict[GsModel.REGISTERED_CODE] == "":
            if len(all_info_dict[GsModel.CODE]) == 18:
                all_info_dict[GsModel.UNIFIED_SOCIAL_CREDIT_CODE] = all_info_dict[GsModel.CODE]
            else:
                all_info_dict[GsModel.REGISTERED_CODE] = all_info_dict[GsModel.CODE]
            return

        if all_info_dict[GsModel.CODE] != "" \
                and all_info_dict[GsModel.UNIFIED_SOCIAL_CREDIT_CODE] != "" \
                and all_info_dict[GsModel.REGISTERED_CODE] != "":
            if len(all_info_dict[GsModel.UNIFIED_SOCIAL_CREDIT_CODE]) != 18:
                all_info_dict[GsModel.UNIFIED_SOCIAL_CREDIT_CODE] = ""
                self.log.error("{code} 统一社会信用号不为18位: {province} {company}".format(
                    code=all_info_dict[GsModel.UNIFIED_SOCIAL_CREDIT_CODE], company=company, province=province))
            return

        self.log.error("有多个注册号信息: {province} {company} {code} {re_code} {un_code}".format(
            province=province,
            company=company,
            code=all_info_dict[GsModel.CODE],
            re_code=all_info_dict[GsModel.REGISTERED_CODE],
            un_code=all_info_dict[GsModel.UNIFIED_SOCIAL_CREDIT_CODE]))
